package zunge.lbe.analysis;

import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileReader;
import java.io.FileWriter;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashSet;
import java.util.List;

import zunge.tools.DbConn;
import zunge.tools.Tools;

public class KMeans {
	public static final int K = 20;
	public static final int N = 40;
	public static final int C = 2;
	public static final int F = 10; //Top-F features in the report
	
	public String folder;
	public String filePrefix;
	public int k;
	public static final String fsuf_center = "_center.txt";
	public static final String fsuf_label = "_label.txt";
	public static final String fsuf_size = "_size.txt";
	public static final String fsuf_log = "_log.txt";
	public static final String fsuf_report = "_report.txt";
	public static final String fsuf_heatmap = "_heatmap.txt";
	public static final String fsuf_stderrmap = "_stderrmap.txt";
	public static final String fsuf_llresult = "_llresult.txt";
	public static final String fsuf_qa = "_qa.txt";
	public String getFile(String label){
		return folder+filePrefix+label;
	}
	
	public KMeans(String folder, String filePrefix, int k){
		this.folder = folder;
		this.filePrefix = filePrefix;
		this.k = k;
	}
	
	public String genScript(String libFolder, String fin_uf) throws Exception{
		new File(getFile(fsuf_center)).delete();
		new File(getFile(fsuf_label)).delete();
		new File(getFile(fsuf_size)).delete();
		new File(getFile(fsuf_log)).delete();
		
		return "R --slave --args"
				+" "+k+" "+N+" "+C
				+" "+fin_uf
				+" "+getFile(fsuf_center)+" "+getFile(fsuf_label)
				+" "+getFile(fsuf_size)+" "+getFile(fsuf_log)
				+" < "+libFolder+"rscript/kmeans.R";
	}
	
	
	int[] getCLabels(DataSet dataSet) throws Exception{
		int[] clabels = new int[dataSet.uMap.size()];
		BufferedReader br = new BufferedReader(new FileReader(getFile(fsuf_label)));
		for(int u = 0;u < dataSet.uMap.size();++u){clabels[u] = Integer.parseInt(br.readLine());}
		br.close();
		return clabels;
	}

	static final String[] pLabels = new String[]{
		"Send SMS","Phone Call","SMS DB","Contact","Call Logs","Positioning",
		"Phone ID","3G Network","Wi-Fi Network","ROOT Privileges","Phone State","Call Monitoring"
	};
	/*
	 * Send SMS
Phone Call
SMS DB
Contact
Call Logs
Positioning
Phone ID
3G Network
Wi-Fi Network
ROOT Privileges
Phone State
Call Monitoring
	}
	 */
	
	/** Generate report of top-F discriminative features */
	public void genReport(DbConn conn, String f_matrix, DataSet dataSet) throws Exception{
		
		//Generate Temporary Table on SQL Database
		String query = "drop table if exists tmp_cupd";
		conn.executeUpdate(conn.getPreparedStatement(query));
		query = "create table tmp_cupd (c int, u int, p int, d double, primary key (u,p))";
		conn.executeUpdate(conn.getPreparedStatement(query));
		PreparedStatement ps = conn.getPreparedStatement("insert into tmp_cupd (c,u,p,d) values (?,?,?,?)");
			
		//Read matrix and store into table
		int[] clabels = getCLabels(dataSet);
		BufferedReader br = new BufferedReader(new FileReader(f_matrix));
		int uid,pid;double d;
		String line;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			pid = Integer.parseInt(frags[1]);
			d = Double.parseDouble(frags[2]);
			if(uid > dataSet.uMap.size()){continue;}
			ps.setInt(1, clabels[uid-1]-1);
			ps.setInt(2, uid-1);
			ps.setInt(3, pid-1);
			ps.setDouble(4, d);
			conn.executeUpdate(ps);
		}
		br.close();
		
		/* Search for top-confident discriminative patterns for each cluster */
		//Generate patterns
		List<M4PatternSet> patternSets = new ArrayList<M4PatternSet>();
		//Generate 1-dim patterns
		for(int m = 0;m < 12;++m){
			patternSets.add(new M4PatternSet(conn, k, 1, m, -1));
		}
		//Generate 2-dim patterns
		for(int m1 = 0;m1 < 11;++m1){
			for(int m2 = m1+1; m2 < 12;++m2){
				patternSets.add(new M4PatternSet(conn, k, 2, m1, m2));
			}
		}
			
		//Write Report
		BufferedWriter bw = new BufferedWriter(new FileWriter(getFile(fsuf_report)));
		bw.write("Report for K="+k+"\n");
		
		double totalTopC = 0;
		double totalSizedTopC = 0;
		double totalSize = 0;
		for(int c = 0;c < k;++c){	
			//Generate top-F patterns by F-measure
			List<M4Pattern> patterns = new ArrayList<M4Pattern>();
			for(M4PatternSet patternSet:patternSets){
				//patterns.add(patternSet.getPattern(c, M4Pattern.TYPE_FMEASURE));
				patterns.add(patternSet.getPattern(c, M4Pattern.TYPE_PURITYEDGE));
			}
			Collections.sort(patterns);
				
			bw.write("Cluster "+c+" size="+patterns.get(0).patternSet.cSize[c]+"\n");
				
			totalTopC += patterns.get(0).confidenceValue;
			totalSizedTopC += patterns.get(0).confidenceValue * patterns.get(0).patternSet.cSize[c];
			totalSize += patterns.get(0).patternSet.cSize[c];
			
			for(int i = 0;i < F && i < patterns.size();++i){
				M4Pattern pattern = patterns.get(i);
				bw.write("\tTop "+(i+1)+":\t");
				switch(pattern.d){
				case 1:
					bw.write(getDecision(pattern.d1)+"("+pLabels[pattern.m1]+") "
							+" [C="+getDecimal(pattern.confidenceValue)+"] ");
					break;
				case 2:
					bw.write(getDecision(pattern.d1)+"("+pLabels[pattern.m1]+") "
							+getDecision(pattern.d2)+"("+pLabels[pattern.m2]+") "
							+" [C="+getDecimal(pattern.confidenceValue)+"] ");
					break;
				}
				bw.write("\t"+pattern.precision+"\t"+pattern.recall+"\t"+pattern.purityEdge);
				bw.write("\n");
			}
		}
		bw.write("avg top confidence value = "+totalTopC/k+"\t"+totalSizedTopC/totalSize+"\n");
		
		bw.close();	
	}
	
	String getDecision(int d){
		if(d==1){return "Acc";}
		else if(d==-1){return "Rej";}
		else{return null;}
	}
	String getDecimal(double d){
		return String.format("%.2f", d);
	}
		
		
	class M4PatternSet{
		int D; //depth of pattern search
		int S; //power(2,d)
		int m1;
		int m2;
			
		double[][] cCount;
		double[] totalCount;
		double[] cSize;
			
		double[][] precision; //given cluster, ratio of users satisfying settings.
		double[][] recall; //given settings, ratio of users who fall into this cluster.
		double[][] fmeasure;
		double[][] purityEdge; //considering the percentage of users who satisfy settings, what is the edge of this cluster comparing to average value?
			
		void init(int k){
			cCount = new double[k][S]; for(int i =0;i<k;++i){Arrays.fill(cCount[i], 0);}
			totalCount = new double[S]; Arrays.fill(totalCount, 0);
			cSize = new double[k];Arrays.fill(cSize, 0);
				
			precision = new double[k][S];for(int i =0;i<k;++i){Arrays.fill(precision[i], 0);}
			recall = new double[k][S];for(int i =0;i<k;++i){Arrays.fill(recall[i], 0);}
			fmeasure = new double[k][S];for(int i =0;i<k;++i){Arrays.fill(fmeasure[i], 0);}
			purityEdge = new double[k][S];for(int i =0;i<k;++i){Arrays.fill(purityEdge[i], 0);}
		}
			
		/** d = depth of search, d=1,2.
		 */
		M4PatternSet(DbConn conn, int k, int D, int m1, int m2) throws Exception{
			this.D = D;
			this.S = (int)Math.pow(2, D);
			this.m1 = m1;
			this.m2 = m2;
			init(k);
				
			//Get Original Data
			ResultSet rs;
			for(int c = 0;c < k;++c){
				rs = conn.executeQuery("select count(distinct u) from tmp_cupd where c = "+c);
				rs.next();
				cSize[c] = rs.getDouble(1);
				rs.close();
			}
				
			switch(D){
			case 1:
					for(int s = 0;s < 2;++s){//encoding of settings: 0:-, 1:+
						String criteria = (s==0)?"d < 0":"d >= 0";
						
						totalCount[s] = getUserSet(m1, criteria, -1, conn).size();
						for(int c = 0;c < k;++c){
							cCount[c][s] = getUserSet(m1, criteria, c, conn).size();
						}
					}
					break;
			case 2:
					for(int s = 0;s < 4;++s){//encoding of settings: 0:--, 1:-+, 2:+-, 3:++
						String criteria1 = (s<2)?"d < 0":"d >= 0";
						String criteria2 = (s%2==0)?"d < 0":"d >= 0";
						
						HashSet<Integer> userSet = getUserSet(m1, criteria1, -1, conn);
						//System.err.println("set1.size="+userSet.size());
						userSet.retainAll(getUserSet(m2, criteria2, -1, conn));
						totalCount[s] = userSet.size();
						if(totalCount[s]==0){System.err.println("totalCount=0 D="+D+" s="+s);}
						
						for(int c = 0;c < k;++c){
							userSet = getUserSet(m1, criteria1, c, conn);
							userSet.retainAll(getUserSet(m2, criteria2, c, conn));
							cCount[c][s] = userSet.size();
						}
					}
					break;
				}
				
				
			//Calc stat values
			for(int s = 0;s < S;++s){
				for(int c = 0;c < k;++c){
					if(cSize[c]>0){precision[c][s] = cCount[c][s] / cSize[c];}
					if(totalCount[s]>0){recall[c][s] = cCount[c][s] / totalCount[s];}
						
					if(precision[c][s] + recall[c][s]>0){fmeasure[c][s] = 2*precision[c][s]*recall[c][s] / (precision[c][s] + recall[c][s]);}
				}
			}
			for(int s = 0;s < S;++s){
				for(int c = 0;c < k;++c){
					for(int c2 = 0;c2 < k;++c2){purityEdge[c][s] += (precision[c][s] - precision[c2][s]);}
					if(k>2){purityEdge[c][s] /= (k-1);}
					//{System.out.println("m1="+m1+" m2="+m2+" c="+c+"s="+s+" "+precision[c][s]+" "+recall[c][s]+" "+purityEdge[c][s]);}
				}
			}
		}
		
		/** c=-1 for general statistics */
		HashSet<Integer> getUserSet(int p, String criteria, int c, DbConn conn) throws Exception{
			String query = (c==-1)
					?"select distinct u from tmp_cupd where p = "+p+" and "+criteria
					:"select distinct u from tmp_cupd where p = "+p+" and c = "+c+" and "+criteria;
			HashSet<Integer> userSet = new HashSet<Integer>();
			ResultSet rs = conn.executeQuery(query);
			while(rs.next()){
				userSet.add(rs.getInt(1));
			}
			rs.close();
			return userSet;
		}
			
			
		/** Generate pattern for each cluster on these features (pick the feature settings with max confidence value) */
		M4Pattern getPattern(int c, int type) throws Exception{
			double[][] measure;
			switch(type){
				case M4Pattern.TYPE_FMEASURE:
					measure = fmeasure;break;
				case M4Pattern.TYPE_PURITYEDGE:
					measure = purityEdge;break;
				default:
					throw new Exception("wrong type");
			}
				
					
			int maxS = -1;
			double maxV = -10000;
			for(int s = 0;s < S;++s){if(measure[c][s]>maxV){maxS = s;maxV = measure[c][s];}}
				
			switch(D){
				case 1://encoding of settings: 0:-, 1:+
					switch(maxS){
					case 0: return new M4Pattern(this, type, c, D, m1, -1, -1, -1, measure[c][0], precision[c][0],recall[c][0],purityEdge[c][0]);
					case 1: return new M4Pattern(this, type, c, D, m1, 1, -1, -1, measure[c][1], precision[c][1],recall[c][1],purityEdge[c][1]);
					default: throw new Exception("wrong maxS");
					}
				case 2://encoding of settings: 0:--, 1:-+, 2:+-, 3:++
					switch(maxS){
					case 0: return new M4Pattern(this, type, c, D, m1, -1, m2, -1, measure[c][0], precision[c][0],recall[c][0],purityEdge[c][0]);
					case 1: return new M4Pattern(this, type, c, D, m1, -1, m2, 1, measure[c][1], precision[c][1],recall[c][1],purityEdge[c][1]);
					case 2: return new M4Pattern(this, type, c, D, m1, 1, m2, -1, measure[c][2], precision[c][2],recall[c][2],purityEdge[c][2]);
					case 3: return new M4Pattern(this, type, c, D, m1, 1, m2, 1, measure[c][3], precision[c][3],recall[c][3],purityEdge[c][3]);
					default: throw new Exception("wrong maxS");
					}
				default:
					throw new Exception("wrong D");
			}
		}
	}

		
	class M4Pattern implements Comparable<M4Pattern>{
		public M4PatternSet patternSet;
		public int type;
		public static final int TYPE_FMEASURE = 0;
		public static final int TYPE_PURITYEDGE = 1;
			
		public Double confidenceValue;
		public int c; //clusterId
		public int d; //depth of search
		public int m1,m2;
		public int d1,d2;
			
		public double precision;
		public double recall;
		public double purityEdge;
			
		public M4Pattern(M4PatternSet patternSet, int type, int c, int d, int m1, int d1, int m2, int d2, double cfv, 
					double precision, double recall, double purityEdge){
			this.patternSet = patternSet;
			this.type = type;
			this.c = c;
			this.d = d;
			this.m1 = m1; this.d1 = d1;
			this.m2 = m2; this.d2 = d2;
			this.confidenceValue = cfv;
			this.precision = precision;
			this.recall = recall;
			this.purityEdge = purityEdge;
		}

		@Override
		public int compareTo(M4Pattern arg0) {
			return arg0.confidenceValue.compareTo(this.confidenceValue);
		}
	}
		
		
	
	/*
	void genHeatMap(String f_matrix, DataSet dataSet) throws Exception{
		double[][] prec = new double[k][DataSet.P]; for(int i = 0;i < k;++i){Arrays.fill(prec[i], 0);}
		int[] cSize = new int[k]; Arrays.fill(cSize, 0);
		int[] clabels = getCLabels(dataSet);
		for(int i = 0;i < clabels.length;++i){cSize[clabels[i]-1]++;}
		
		BufferedReader br = new BufferedReader(new FileReader(f_matrix));
		int uid,pid;double d;
		String line;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			pid = Integer.parseInt(frags[1]);
			d = Double.parseDouble(frags[2]);
			if(uid > dataSet.uMap.size()){continue;}
			prec[clabels[uid-1]-1][pid-1] += d;
		}
		br.close();
		for(int c = 0;c < k;++c){
			if(cSize[c]>0){
				for(int p = 0;p < DataSet.P;++p){
					prec[c][p] /= cSize[c];
				}
			}
		}
			
		BufferedWriter bw = new BufferedWriter(new FileWriter(getFile(fsuf_heatmap)));
		for(int c = 0;c < k;++c){
			for(int p = 0;p < DataSet.P;++p){
				bw.write(prec[c][p]+" ");
			}
			bw.write("\n");
		}
		bw.close();
	}
	*/
	void genHeatMap(String f_matrix, DataSet dataSet) throws Exception{
		double[][] avg = new double[k][DataSet.P]; for(int i = 0;i < k;++i){Arrays.fill(avg[i], 0);}
		double[][] err = new double[k][DataSet.P]; for(int i = 0;i < k;++i){Arrays.fill(err[i], 0);}
		
		int[] cSize = new int[k]; Arrays.fill(cSize, 0);
		int[] clabels = getCLabels(dataSet);
		for(int i = 0;i < clabels.length;++i){cSize[clabels[i]-1]++;}
		
		BufferedReader br = new BufferedReader(new FileReader(f_matrix));
		int uid,pid;double d;
		String line;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			pid = Integer.parseInt(frags[1]);
			d = Double.parseDouble(frags[2]);
			if(uid > dataSet.uMap.size()){continue;}
			avg[clabels[uid-1]-1][pid-1] += d;
		}
		br.close();
		for(int c = 0;c < k;++c){
			if(cSize[c]>0){
				for(int p = 0;p < DataSet.P;++p){
					avg[c][p] /= cSize[c];
				}
			}
		}
		br = new BufferedReader(new FileReader(f_matrix));
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			pid = Integer.parseInt(frags[1]);
			d = Double.parseDouble(frags[2]);
			if(uid > dataSet.uMap.size()){continue;}
			double dif = (d - avg[clabels[uid-1]-1][pid-1]);
			err[clabels[uid-1]-1][pid-1] += dif*dif;
		}
		br.close();
		for(int c = 0;c < k;++c){
			if(cSize[c]>0){
				for(int p = 0;p < DataSet.P;++p){
					err[c][p] = err[c][p] / cSize[c];//Math.sqrt(err[c][p] / cSize[c]);
				}
			}
		}
		
		//Write Map
		BufferedWriter bw = new BufferedWriter(new FileWriter(getFile(fsuf_heatmap)));
		for(int c = 0;c < k;++c){
			for(int p = 0;p < DataSet.P;++p){bw.write(avg[c][p]+" ");}
			bw.write("\n");
		}
		bw.close();
		bw = new BufferedWriter(new FileWriter(getFile(fsuf_stderrmap)));
		for(int c = 0;c < k;++c){
			for(int p = 0;p < DataSet.P;++p){bw.write(err[c][p]+" ");}
			bw.write("\n");
		}
		bw.close();
	}
	
	void genQA(String f_matrix, DataSet dataSet) throws Exception{
		//TODO
		int[] clabels = getCLabels(dataSet);
		
		double[][] matrix = new double[clabels.length][DataSet.P];
		
		BufferedReader br = new BufferedReader(new FileReader(f_matrix));
		int uid,pid;double d;
		String line;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			pid = Integer.parseInt(frags[1]);
			d = Double.parseDouble(frags[2]);
			if(uid > dataSet.uMap.size()){continue;}
			matrix[uid-1][pid-1] = d;
		}
		br.close();
		for(int u = 0;u < clabels.length;++u){
			for(int p = 0;p < DataSet.P;++p){
				
			}
		}
		double t = 0.8;
		BufferedWriter bw = new BufferedWriter(new FileWriter(getFile(fsuf_qa)));
		for(int u = 0;u < clabels.length;++u){
			bw.write(clabels[u]+"");
			for(int p = 0;p < DataSet.P;++p){
				int value = 0;
				if(matrix[u][p]>t){value=1;}
				if(matrix[u][p]<-t){value=-1;}
				bw.write(" "+value);
			}
			bw.write("\n");
		}
		bw.close();
	}
	
	
	
	/** Test on:  {c, a, p, [c,p] } */
	public List<String> genTest(String libFolder, DataSet dataSet) throws Exception{
		//Read cluster labels
		int[] clabels = getCLabels(dataSet);
		
		//Read Matrix
		double[][] acc = new double[k][DataSet.P];
		double[][] rej = new double[k][DataSet.P];
		double[][] ratio = new double[k][DataSet.P];
		for(int kk = 0;kk < k;++kk){Arrays.fill(acc[kk], 0);Arrays.fill(rej[kk], 0);Arrays.fill(ratio[kk], 0);}
		BufferedReader br = new BufferedReader(new FileReader(dataSet.getFile(dataSet.fsuf_recordUAP)));
		String line; int uid,pid,did;
		while((line=br.readLine())!=null){
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			pid = Integer.parseInt(frags[2]);
			did = Integer.parseInt(frags[3]);
			if(did==1){acc[clabels[uid-1]-1][pid-1]++;}
			else if(did==-1){rej[clabels[uid-1]-1][pid-1]++;}
		}
		br.close();
		for(int kk = 0;kk < k;++kk){
			for(int p = 0;p < DataSet.P;++p){
				ratio[kk][p] = (acc[kk][p]+rej[kk][p]==0)?0:( (acc[kk][p]-rej[kk][p]) / (acc[kk][p]+rej[kk][p]) );
			}
		}
		
		//Generate Inputs and Scripts
		List<String> scripts = new ArrayList<String>();
		for(DataSetFold dataSetFold:dataSet.folds){
			//Generate converted input
			String f_train = dataSetFold.getFile(dataSetFold.fsuf_train);
			String f_test = dataSetFold.getFile(dataSetFold.fsuf_test);
			String f_ctrain =  f_train+"_k"+k+".txt";
			String f_ctest =  f_test+"_k"+k+".txt";
			genInput(f_train,f_ctrain, clabels, ratio);
			genInput(f_test,f_ctest, clabels, ratio);
			LibLinear ll = new LibLinear(folder, filePrefix+"_liblinear_v"+dataSetFold.v);
			ll.genData(f_ctrain, f_ctest, 
					new int[]{LibLinear.FLAG_NOMINAL,LibLinear.FLAG_NOMINAL,LibLinear.FLAG_NOMINAL, LibLinear.FLAG_NUMERIC, LibLinear.FLAG_NUMERIC}, 
					new boolean[]{true,true,true,true,true});
			scripts.add(ll.genScript(libFolder));
		}
		
		return scripts;
	}
	void genInput(String fin, String fout, int[] clabels, double[][] ratio) throws Exception{
		BufferedReader br = new BufferedReader(new FileReader(fin));
		BufferedWriter bw = new BufferedWriter(new FileWriter(fout));
		String line; int uid,aid,pid,did,cid; double r;
		while((line=br.readLine())!=null){//Input: uid aid pid did,  Output: cid aid pid [cid,pid] did
			String[] frags = line.split(" ");
			uid = Integer.parseInt(frags[0]);
			pid = Integer.parseInt(frags[2]);
			cid = clabels[uid-1];
			r = ratio[cid-1][pid-1];
			bw.write(cid+" "+frags[1]+" "+pid+" "+r+" "+frags[3]+"\n");
		}
		br.close();
		bw.close();
	}
	
	public void getTestResult(String libFolder, DataSet dataSet) throws Exception{
		double[] acc = new double[DataSet.V];
		for(DataSetFold dataSetFold:dataSet.folds){
			LibLinear ll = new LibLinear(folder, filePrefix+"_liblinear_v"+dataSetFold.v);
			acc[dataSetFold.v] = ll.getAccuracy();
		}
		BufferedWriter bw = new BufferedWriter(new FileWriter(getFile(fsuf_llresult)));
		bw.write(Tools.getAverage(acc)+" "+Tools.getVariance(acc)+"\n");
		bw.close();
	}
}
